import random
import time

import requests
from lxml import etree
from app.models import *
from app import db, app
'''sql文件里有数据了，可以不用这个'''
'''
获取房屋信息，地址信息和用户信息的脚本，直接运行该文件填充address,house,user表(不用启动项目)
(数量有点多，可以运行一半直接停掉)
运行前:
    完成config.py文件下的数据库配置
    创建address,house,user表(SQL脚本在sql包下)
    下载相应的python包，包信息在requirements.txt文件下(pip install -r requirements.txt)
运行完:
    delete from house where img_urls like '%https://pic2.58cdn.com.cn/nowater/fangfe/n_v2a37a7b59b5444395940bbca67bb88407.png%';
    执行上面的sql语句去除无效图片的房屋信息
'''
# User - Agent 池
user_agent_pool = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 SLBrowser/9.0.6.2081 SLBChan/111 SLBVPV/64-bit',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.0'
]


def get_random_user_agent():
    return random.choice(user_agent_pool)


headers = {
    'user-agent': get_random_user_agent()
}
response = requests.get('https://bj.zu.anjuke.com/', headers=headers)

tree = etree.HTML(response.text)

city_urls = {

}
xp1 = tree.xpath('/html/body/div[2]/div/div[1]/div[2]/dl')

# 获取城市-网址键值对
for i in xp1:
    new_xp = i.xpath('dd/a')
    for j in new_xp:
        city_urls[j.xpath('text()')[0]] = j.xpath('@href')[0]
print(city_urls)

for city, url in city_urls.items():
    res = requests.get(url, headers=headers).text
    new_tree = etree.HTML(res)
    xp2 = new_tree.xpath('/html/body/div[5]/div[3]/div[1]/div')
    for index, xp in enumerate(xp2):
        if index < 3:
            continue
        try:
            # 详细信息的网址
            detail_url = xp.xpath('a/@href')[0]
            img_urls=[]
            new_headers = {
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 SLBrowser/9.0.6.2081 SLBChan/111 SLBVPV/64-bit',
                'referrer': url
            }
            r=requests.get(detail_url,headers=headers).text
            # print(r)
            new_tree2=etree.HTML(r)
            xpx=new_tree2.xpath('/html/body/div[3]/div[2]/div[1]/div[1]/div[1]/div/div[3]/div[2]/div')
            for i,x in enumerate(xpx):
                if i==0:
                    continue
                img_urls.append(x.xpath('img/@data-src')[0])
            # img_url = xp.xpath('a/img/@src')[0]
            if len(img_urls)==0:
                img_urls.append(xp.xpath('a/img/@src')[0])
            title = xp.xpath('div[1]/h3/a/b/text()')[0]
            base_information = xp.xpath('div[1]/p[1]/b[1]/text()')[0] + "室" + xp.xpath('div[1]/p[1]/b[2]/text()')[
                0] + "厅"
            area = xp.xpath('div[1]/p[1]/b[3]/text()')[0]
            district = xp.xpath('div[1]/address/a/following-sibling::text()[1]')[0].replace('\n', '').replace(' ',
                                                                                                              '').strip()
            detail_address = (xp.xpath('div[1]/address/span[2]/following-sibling::text()[1]')[0] +
                              xp.xpath('div[1]/address/a/text()')[0]).replace('\n', '').replace(' ', '')
            xp3 = xp.xpath('div[1]/p[2]/span')
            descriptor = []
            for j in xp3:
                descriptor.append(j.xpath('text()')[0])
            landlord = xp.xpath('div[1]/p[3]/span[1]/text()')[0]
            price = xp.xpath('div[2]/strong/text()')[0]
            deposit=int(price*1.25) if int(price) > 5000 else int(price)*2
            print(detail_url, img_urls, title, base_information, area, district, detail_address, descriptor, landlord,
                  price,
                  sep='\n')
            print('------------------------')
            with app.app_context():
                user = User.query.filter_by(username=landlord).first()
                if user is None:
                    user = User(
                        username=landlord,
                        password='e10adc3949ba59abbe56e057f20f883e',
                        identity=3
                    )
                    db.session.add(user)
                    db.session.commit()
                landlord_id = User.query.filter_by(username=landlord).first().id

                # address = Address.query.filter_by(city=city, district=district, detailed_address=detail_address).first()
                # if address is None:
                #     address = Address(
                #         city=city,
                #         district=district,
                #         detailed_address=detail_address
                #     )
                #     db.session.add(address)
                #     db.session.commit()
                address = Address(
                    city=city,
                    district=district,
                    detailed_address=detail_address
                )
                db.session.add(address)
                db.session.commit()


                # address_id = Address.query.filter_by(city=city, district=district,
                #                                      detailed_address=detail_address).first().id
                address_id=address.id

                house = House(
                    title=title,
                    house_type=base_information,
                    area=area,
                    landlord_id=landlord_id,
                    address_id=address_id,
                    price=int(price),
                    img_urls=str(img_urls),
                    descriptor=str(descriptor),
                    deposit=deposit,
                    status=2
                )
                db.session.add(house)
                db.session.commit()
        except Exception as e:
            print(e)
            continue
        # 添加请求间隔，单位为秒
        time.sleep(random.uniform(1, 3))